TFM:Impacto de la calidad del aire en la salud de las personas en la ciudad de Madrid¶
In [310]:
%%HTML
<script src="require.js"></script>
In [2]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.io as pio
pio.renderers.default='notebook'
import seaborn as sns
import time
import datetime
from sklearn.preprocessing import LabelEncoder,OneHotEncoder,MinMaxScaler,StandardScaler,RobustScaler
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
from statsmodels.tsa.seasonal import seasonal_decompose
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split,GridSearchCV,RandomizedSearchCV
from sklearn.metrics import mean_squared_error,mean_absolute_error,mean_absolute_percentage_error
from catboost import CatBoostRegressor
import shap
from lightgbm import LGBMRegressor
In [312]:
pd.set_option('display.max_columns',None)
In [313]:
df=pd.read_excel('data/CMBD_6_20181217-135856.xlsx')
In [314]:
df_calidad_aire_07=pd.read_csv('data/datos07.csv',sep=';')
df_calidad_aire_08=pd.read_csv('data/datos08.csv',sep=';')
df_calidad_aire_09=pd.read_csv('data/datos09.csv',sep=';')
df_calidad_aire_10=pd.read_csv('data/datos10.csv',sep=';')
df_calidad_aire_11=pd.read_csv('data/datos11.csv',sep=';')
df_calidad_aire_12=pd.read_csv('data/datos12.csv',sep=';')
df_calidad_aire_13=pd.read_csv('data/datos13.csv',sep=';')
df_calidad_aire_14=pd.read_csv('data/datos14.csv',sep=';')
df_calidad_aire_15=pd.read_csv('data/datos15.csv',sep=';')
df_calidad_aire=pd.concat([df_calidad_aire_07, df_calidad_aire_08,df_calidad_aire_09,df_calidad_aire_10,df_calidad_aire_11,df_calidad_aire_12,df_calidad_aire_13,df_calidad_aire_14,df_calidad_aire_15],axis=0,ignore_index=True)
In [315]:
df_temperatura_07=pd.read_csv('data/temp_2007.csv')
df_temperatura_08=pd.read_csv('data/temp_2008.csv')
df_temperatura_09=pd.read_csv('data/temp_2009.csv')
df_temperatura_10=pd.read_csv('data/temp_2010.csv')
df_temperatura_11=pd.read_csv('data/temp_2011.csv')
df_temperatura_12=pd.read_csv('data/temp_2012.csv')
df_temperatura_13=pd.read_csv('data/temp_2013.csv')
df_temperatura_14=pd.read_csv('data/temp_2014.csv')
df_temperatura_15=pd.read_csv('data/temp_2015.csv')
df_temperatura=pd.concat([df_temperatura_07, df_temperatura_08,df_temperatura_09,df_temperatura_10,df_temperatura_11,df_temperatura_12,df_temperatura_13,df_temperatura_14,df_temperatura_15],axis=0,ignore_index=True)
Análisis exploratorio¶
In [316]:
df.head()
Out[316]:
In [317]:
df.tail()
Out[317]:
In [318]:
df.columns=df.columns.str.strip().str.lower().str.replace(' ','_')
In [319]:
df.año.sort_values().unique()
Out[319]:
In [320]:
df.describe(include='all').T
Out[320]:
In [321]:
comunidad_autonoma=13
provincia=28
In [322]:
df.drop(['comunidad_autónoma','provincia'],axis=1,inplace=True)
In [323]:
df_calidad_aire.head()
Out[323]:
In [324]:
df_temperatura.head()
Out[324]:
In [325]:
df.dtypes
Out[325]:
In [326]:
df_calidad_aire.dtypes
Out[326]:
In [327]:
df_temperatura.dtypes
Out[327]:
In [328]:
df.shape
Out[328]:
In [329]:
df_calidad_aire.shape
Out[329]:
In [330]:
df_temperatura.shape
Out[330]:
In [331]:
df.isnull().mean().sort_values(ascending=False)*100
Out[331]:
In [332]:
df_calidad_aire.isnull().mean().sort_values(ascending=False)*100
Out[332]:
In [333]:
df_temperatura.isnull().mean().sort_values(ascending=False)*100
Out[333]:
In [334]:
df.duplicated().sum()
Out[334]:
In [335]:
df.select_dtypes('object').apply(pd.Series.nunique, axis = 0).sort_values()
Out[335]:
In [336]:
def vuelta_fecha(x):
x=str(x)
if len(x)==8:
año=x[4:]
mes=x[2:4]
dia=x[:2]
fecha=año+'-'+mes+'-'+dia
else:
año=x[3:]
mes=x[1:3]
dia=x[:1]
fecha=año+'-'+mes+'-'+'0'+dia
return pd.to_datetime(fecha,format='%Y-%m-%d')
In [337]:
df.fecha_de_ingreso=df.fecha_de_ingreso.apply(lambda x:vuelta_fecha(x))
In [338]:
df['año_ingreso']=df.fecha_de_ingreso.dt.year
df['mes_ingreso']=df.fecha_de_ingreso.dt.month
df['dia_ingreso']=df.fecha_de_ingreso.dt.day
df['admission_date'] = df.fecha_de_ingreso
In [339]:
plt.subplots(figsize=(20,20))
sns.heatmap(df.corr())
Out[339]:
In [340]:
df.select_dtypes('number').describe()
Out[340]:
In [341]:
df.grupo_hospital.value_counts(dropna=False)
Out[341]:
In [342]:
df.grupo_cluster.value_counts(dropna=False)
Out[342]:
In [343]:
df.provincia_residencia.value_counts(dropna=False)
Out[343]:
In [344]:
df.régimen_financiación.value_counts(dropna=False)
Out[344]:
In [345]:
df.tipo_ingreso.value_counts(dropna=False)
Out[345]:
In [346]:
df.tipo_alta.value_counts(dropna=False)
Out[346]:
In [347]:
conditionlist = [
(df['edad'] <= 12) ,
(df['edad'] > 12) & (df['edad'] <=18),
(df['edad'] >18) & (df['edad'] <=26),
(df['edad'] >=27) & (df['edad'] <=59),
(df['edad']>=60)]
choicelist = ['Niño','Adolescente', 'Joven', 'Adulto','Persona Mayor']
df['grupo_edad'] = np.select(conditionlist, choicelist, default='Not Specified')
In [348]:
df[['grupo_edad','estancia']].boxplot(by='grupo_edad',figsize=(12,10))
Out[348]:
In [349]:
df.corr()['edad'].sort_values(ascending=False)
Out[349]:
In [350]:
df.corr()['estancia'].sort_values(ascending=False)
Out[350]:
In [351]:
df.corr()['coste'].sort_values(ascending=False)
Out[351]:
In [352]:
df.corr()['valor_peso_español'].sort_values(ascending=False)
Out[352]:
In [353]:
df['ingreso_mes_año']=df.fecha_de_ingreso.dt.strftime('%m-%Y')
In [354]:
df.ingreso_mes_año=pd.to_datetime(df.ingreso_mes_año)
In [355]:
px.line(df.groupby('ingreso_mes_año').count().reset_index().sort_values('ingreso_mes_año').rename(columns={"fecha_de_ingreso": "numero de ingresos"}),x='ingreso_mes_año',y="numero de ingresos")
In [356]:
features=['grupo_cluster','sexo','fecha_de_ingreso','admission_date','dia_ingreso','mes_ingreso','año_ingreso','tipo_ingreso','estancia','historia_recodificada','cip_recodificado','diagnóstico_principal','diagnóstico_2','procedimiento_1','valor_peso_español','nivel_severidad','nivel_mortalidad','servicio','edad','reingreso','coste']
In [357]:
len(features)
Out[357]:
In [358]:
df[['tipo_ingreso','edad']].boxplot(by='tipo_ingreso',figsize=(12,10))
Out[358]:
In [359]:
df[['estancia','sexo']].boxplot(by='sexo',figsize=(12,10))
Out[359]:
In [360]:
px.bar(df[['nivel_severidad','sexo']].groupby(by='sexo').value_counts().reset_index().set_index(['sexo','nivel_severidad']).reset_index().rename(columns={0:'valor'}),x='nivel_severidad',y='valor',color='sexo')
In [361]:
df.ingreso_mes_año.value_counts()
Out[361]:
In [362]:
df[features].head()
Out[362]:
In [363]:
df=df[features]
In [364]:
plt.subplots(figsize=(20,12))
sns.heatmap(df.corr())
Out[364]:
In [365]:
df_def=pd.read_csv('data/df_definitivo.csv',parse_dates=['fecha'])
df_serie_temporal = df_def.loc[:,['fecha','numero_de_ingresos']]
df_serie_temporal['fecha'] = pd.to_datetime(df_serie_temporal['fecha'])
df_serie_temporal.set_index(keys='fecha',inplace=True)
df_serie_temporal.dropna(inplace=True)
estacionalidad = seasonal_decompose(
df_serie_temporal['numero_de_ingresos'],
model='multiplicative',
period=12
)
fig = estacionalidad.plot()
fig.set_size_inches(10,10)
fig.tight_layout()
plt.show()
Calidad del aire¶
In [366]:
Ds = df_calidad_aire.columns[[x.startswith('D') for x in df_calidad_aire.columns]]
In [367]:
df_d = df_calidad_aire[Ds]
In [368]:
df_d.fillna(0,inplace=True)
In [369]:
df_calidad_aire['value'] = df_d.mean(axis=1)
In [370]:
df_calidad_aire = df_calidad_aire[['PUNTO_MUESTREO','MAGNITUD','ANO','MES','value']]
In [371]:
df_calidad_aire['fecha'] = df_calidad_aire['ANO'].astype(str) + '-' + df_calidad_aire['MES'].astype(str)
In [372]:
df_calidad_aire['fecha'] = pd.to_datetime(df_calidad_aire['fecha'])
In [373]:
df_magnitud = df_calidad_aire.groupby(['MAGNITUD','fecha']).mean().reset_index()
In [374]:
px.line(df_magnitud,x='fecha',y='value',color='MAGNITUD')
In [375]:
magnitud_nombre={1:'Dióxido de Azufre',6:'Monóxido de Carbono',
7:'Monóxido de Nitrógeno', 8:'Dióxido de Nitrógeno',
9:'PartÃculas menor 2.5 μm', 10:'PartÃculas menor 10 μm',
12:'Óxidos de Nitrógeno', 14:'Ozono', 20:'Tolueno',
30:'Benceno', 35:'Etilbenceno', 42:'Hidrocarburos totales (hexano)',
43:'Metano', 44:'Hidrocarburos no metánicos (hexano)'}
In [376]:
def set_values(row, value):
return value[row]
In [377]:
df_magnitud['MAGNITUD_NOMBRE']=df_magnitud.MAGNITUD.apply(set_values,args=(magnitud_nombre,))
In [378]:
px.line(df_magnitud.reset_index(),x='fecha',y='value',color='MAGNITUD_NOMBRE')
In [379]:
df_umbrales = pd.read_excel('data/Umbrales.xlsx')
In [380]:
df_fin = df_magnitud.merge(df_umbrales, left_on='MAGNITUD',right_on='CODIGO', how='inner')
In [381]:
particulas=['Dióxido de Azufre','Monóxido de Carbono','Dióxido de Nitrógeno','PartÃculas menor 2,5 μm','PartÃculas menor 10 μm','Óxidos de Nitrógeno','Ozono','Benceno']
In [382]:
df_fin.drop(['CODIGO','MAGNITUD_y'],axis=1,inplace=True)
In [383]:
df_fin.columns = ['CODIGO','fecha','ANO','MES', 'value', 'PARTICULA', 'ABREVIATURA','MEDIDA','LIMITE', 'COMENTARIO']
In [384]:
df_fin.loc[(df_fin.CODIGO==12) & (df_fin.MES==8) & (df_fin.ANO==2013),'value'] = np.nan
df_fin.loc[(df_fin.CODIGO==12) & (df_fin.MES==8) & (df_fin.ANO==2013),'value'] = df_fin.loc[(df_fin.CODIGO==12) & (df_fin.MES==8)].value.mean()
In [385]:
df_fin.sample(20)
Out[385]:
Datos Climatológicos¶
In [386]:
df_temperatura=df_temperatura[['fecha','tm_mes','hr','p_max','p_mes']]
In [387]:
df_temperatura.p_max=df_temperatura.p_max.str.slice(0,3).astype('float')
In [388]:
df_temperatura.fecha=pd.to_datetime(df_temperatura.fecha)
In [389]:
df_temperatura.tm_mes.interpolate(method="akima",inplace=True)
df_temperatura.hr.interpolate(method="akima",inplace=True)
df_temperatura.p_max.interpolate(method="akima",inplace=True)
df_temperatura.p_mes.interpolate(method="akima",inplace=True)
In [390]:
px.line(df_temperatura,x='fecha',y=['p_max','hr','p_mes','tm_mes'])
In [391]:
df_temperatura.rename(columns={'tm_mes':'temperatura_media_mes','hr':'humedad_relativa','p_max':'precipitaciones','p_mes':'presion_atmosferica_mes'},inplace=True)
Diagnósticos¶
In [392]:
enfermedad_pulmonar_obstructiva_cronica = [
'491.0', '491.1', '491.20', '491.21', '491.22', '491.8', '491.9', '492.0', '492.8', '494', '494.0', '494.1', '496'
]
bronquitis_aguda = [
'466.0','490'
]
asma = [
'493.00', '493.01', '493.02', '493.10', '493.11', '493.12', '493.20', '493.21', '493.22', '493.81', '493.82', '493.90', '493.91', '493.92'
]
fibrosis_quistica_o_anomalias_sistema_respiratorio = [
'277.00', '277.01', '277.02','277.03', '277.09', '516.61','516.62', '516.63', '516.64', '516.69', '747.21', '748.3', '748.4', '748.5', '748.60', '748.61', '748.69', '748.8', '748.9', '750.3', '759.3','770.7'
]
insuficiencia_cardiaca_congestiva = [
'398.91', '402.01', '402.11', '402.91', '404.01', '404.03', '404.11', '404.13', '404.91', '404.93', '428.0', '428.1', '428.20', '428.21', '428.22', '428.23', '428.30', '428.31', '428.32', '428.33', '428.40', '428.41', '428.42', '428.43', '428.9'
]
hipertension_arterial = [
'401.0', '401.9', '402.00', '402.10', '402.90', '403.00', '403.10', '403.90', '404.00', '404.10', '404.90'
]
neumonia_bacteriana = [
'481', '482.2', '482.30', '482,31', '482.32', '482.39', '482.41', '482.42', '482.9', '483.0', '483.1', '483.8', '485', '486'
]
anemia_falciforme_o_enfermedad_hemoglobina_S = [
'282.41', '282.42', '282.60', '282.61', '282.62', '282.63', '282.64', '282.68', '282.69'
]
diabetes_incontrolada_sin_complicacion = [
'250.02','250.03'
]
diabetes_con_complicacion_aguda = [
'250.10', '250,11', '250.12', '250.13', '250.20', '250.21','250.22','250.23', '250.30', '250.31', '250.32', '250.33'
]
diabetes_con_complicacion_cronica = [
'250.40', '250.41', '250.42', '250.43', '250.50', '250.51', '250.52', '250.53', '250.60', '250.61', '250.62', '250.63', '250.70', '250.71', '250.72', '250.73', '250.80','250.81', '250.82', '250.83', '250.90', '250.91', '250.92', '250.93'
]
infarto_agudo_de_miocardio = [
'410.01','410.11','410.21','410.31','410.41','410.51','410.71','410.81','410.90','410.91'
]
Procedimientos¶
In [393]:
dialisis = [
'38.95', '39.27', '39.29', '39.42', '39.43', '39.93', '39.94'
]
Enfermedades¶
In [394]:
pqi_01_complicaciones_a_corto_plazo_de_la_diabetes = (
(df['diagnóstico_principal'].isin(diabetes_con_complicacion_aguda))
&
(df['edad'] >= 15)
)
pqi_03_complicaciones_a_largo_plazo_de_la_diabetes = (
(df['diagnóstico_principal'].isin(diabetes_con_complicacion_cronica))
&
(df['edad'] >= 15)
)
pqi_05_enfermedad_pulmonar_obstructiva_cronica_o_asma_en_mayores = (
(
(df['diagnóstico_principal'].isin(enfermedad_pulmonar_obstructiva_cronica)) |
((df['diagnóstico_principal'].isin(bronquitis_aguda)) & (df['diagnóstico_2'].isin(enfermedad_pulmonar_obstructiva_cronica))) |
(df['diagnóstico_principal'].isin(asma))
)
&
(df['edad'] >= 40)
&
~(df['diagnóstico_2'].isin(fibrosis_quistica_o_anomalias_sistema_respiratorio))
)
pqi_07_hipertension_arterial = (
(df['diagnóstico_principal'].isin(hipertension_arterial))
&
(df['edad'] >= 15)
&
(df['procedimiento_1'].isin(dialisis))
)
pqi_08_insuficiencia_cardiaca_congestiva = (
(df['diagnóstico_principal'].isin(insuficiencia_cardiaca_congestiva))
&
(df['edad'] >= 15)
)
pqi_11_neumonia_bacteriana = (
(df['diagnóstico_principal'].isin(neumonia_bacteriana))
&
(df['edad'] >= 15)
&
~(df['diagnóstico_2'].isin(anemia_falciforme_o_enfermedad_hemoglobina_S))
)
pqi_14_diabetes_incontrolada_sin_complicacion = (
(df['diagnóstico_principal'].isin(diabetes_incontrolada_sin_complicacion))
&
(df['edad'] >= 15)
)
pqi_15_asma_en_adultos_jovenes_15_39_anos = (
(df['diagnóstico_principal'].isin(asma))
&
((df['edad'] >= 18) & (df['edad'] < 39))
&
~ (
(df['diagnóstico_principal'].isin(fibrosis_quistica_o_anomalias_sistema_respiratorio))
|
(df['diagnóstico_2'].isin(fibrosis_quistica_o_anomalias_sistema_respiratorio))
)
)
hospitalizacion_por_infarto_agudo_de_miocardio = (
df['diagnóstico_principal'].isin(infarto_agudo_de_miocardio)
)
Condiciones crónicas¶
In [395]:
complicaciones_a_corto_plazo_de_la_diabetes_en_poblacion_infantil = (
(df['diagnóstico_principal'].isin(diabetes_con_complicacion_aguda))
&
((df['edad'] >= 6) & (df['edad'] < 15))
)
hospitalizacion_por_asma_en_poblacion_infantil = (
(df['diagnóstico_principal'].isin(asma))
&
((df['edad'] >= 2) & (df['edad'] < 15))
&
~(
(df['diagnóstico_principal'].isin(fibrosis_quistica_o_anomalias_sistema_respiratorio))
|
(df['diagnóstico_2'].isin(fibrosis_quistica_o_anomalias_sistema_respiratorio))
)
)
In [396]:
diagnosis_dict = {
'enfermedad pulmonar obstructiva cronica': enfermedad_pulmonar_obstructiva_cronica,
'bronquitis aguda': bronquitis_aguda,
'asma': asma,
'fibrosis quistica o anomalias sistema respiratorio': fibrosis_quistica_o_anomalias_sistema_respiratorio,
'insuficiencia cardiaca congestiva': insuficiencia_cardiaca_congestiva,
'hipertension arterial': hipertension_arterial,
'neumonia bacteriana': neumonia_bacteriana,
'anemia falciforme o enfermedad hemoglobina S': anemia_falciforme_o_enfermedad_hemoglobina_S,
'diabetes incontrolada sin complicacion': diabetes_incontrolada_sin_complicacion,
'diabetes con complicacion aguda': diabetes_con_complicacion_aguda,
'diabetes con complicacion cronica': diabetes_con_complicacion_cronica,
'infarto agudo de miocardio': infarto_agudo_de_miocardio
}
for diagnosis, diagnosis_codes in diagnosis_dict.items():
df.loc[
df['diagnóstico_principal'].isin(diagnosis_codes),
'diagnosis_name'
] = f'{diagnosis}'
df[['diagnóstico_principal','diagnosis_name']].head()
Out[396]:
In [397]:
diagnosticos=list(diagnosis_dict.keys())
diagnosticos.append('fecha')
del diagnosticos[8]
In [398]:
diseases_filters_dict = {
'PQI 01 complicaciones a corto plazo de la diabetes': pqi_01_complicaciones_a_corto_plazo_de_la_diabetes,
'PQI 03 complicaciones a largo plazo de la diabetes': pqi_03_complicaciones_a_largo_plazo_de_la_diabetes,
'PQI 05 enfermedad pulmonar obstructiva cronica o asma en mayores': pqi_05_enfermedad_pulmonar_obstructiva_cronica_o_asma_en_mayores,
'PQI 07 hipertension arterial': pqi_07_hipertension_arterial,
'PQI 08 insuficiencia cardiaca congestiva': pqi_08_insuficiencia_cardiaca_congestiva,
'PQI 11 neumonia bacteriana': pqi_11_neumonia_bacteriana,
'PQI 14 diabetes incontrolada sin complicacion': pqi_14_diabetes_incontrolada_sin_complicacion,
'PQI 15 asma en adultos jovenes 15 39 anos': pqi_15_asma_en_adultos_jovenes_15_39_anos,
'hospitalizacion por infarto agudo de miocardio': hospitalizacion_por_infarto_agudo_de_miocardio,
'complicaciones a corto plazo de la diabetes en poblacion infantil': complicaciones_a_corto_plazo_de_la_diabetes_en_poblacion_infantil,
'hospitalizacion por asma en poblacion infantil': hospitalizacion_por_asma_en_poblacion_infantil
}
for disease, disease_filter in diseases_filters_dict.items():
df.loc[disease_filter, 'disease_name'] = f'{disease}'
In [399]:
campos_de_agrupacion = ['admission_date','diagnosis_name']
campos_de_registro_unico = ['historia_recodificada','cip_recodificado','fecha_de_ingreso']
fig = px.line(
data_frame = df[campos_de_agrupacion + campos_de_registro_unico].groupby(campos_de_agrupacion).count().reset_index().rename(columns={"fecha_de_ingreso": "numero de ingresos"}),
x = 'admission_date',
y = 'numero de ingresos',
color = 'diagnosis_name',
title = 'Ingresos por diagnóstico',
width = 1500, height = 800
)
fig.update_layout(hovermode="x")
In [400]:
df['ingreso_mes_año']=df.fecha_de_ingreso.dt.strftime('%m-%Y')
df.ingreso_mes_año=pd.to_datetime(df.ingreso_mes_año)
In [401]:
campos_de_agrupacion = ['ingreso_mes_año','diagnosis_name']
campos_de_registro_unico = ['historia_recodificada','cip_recodificado','fecha_de_ingreso']
fig = px.line(
data_frame = df[campos_de_agrupacion + campos_de_registro_unico].groupby(campos_de_agrupacion).count().reset_index().rename(columns={"fecha_de_ingreso": "numero de ingresos"}),
x = 'ingreso_mes_año',
y = 'numero de ingresos',
color = 'diagnosis_name',
title = 'Ingresos por diagnóstico',
width = 1500, height = 800
)
fig.update_layout(hovermode="x")
In [402]:
campos_de_agrupacion = ['admission_date','disease_name']
campos_de_registro_unico = ['historia_recodificada','cip_recodificado','fecha_de_ingreso']
fig = px.line(
data_frame = df[campos_de_agrupacion + campos_de_registro_unico].groupby(campos_de_agrupacion).count().reset_index().rename(columns={"fecha_de_ingreso": "numero de ingresos"}),
x = 'admission_date',
y = 'numero de ingresos',
color = 'disease_name',
title = 'Ingresos por enfermedad',
width = 1500, height = 800
)
fig.update_layout(hovermode="x")
In [403]:
campos_de_agrupacion = ['ingreso_mes_año','disease_name']
campos_de_registro_unico = ['historia_recodificada','cip_recodificado','fecha_de_ingreso']
fig = px.line(
data_frame = df[campos_de_agrupacion + campos_de_registro_unico].groupby(campos_de_agrupacion).count().reset_index().rename(columns={"fecha_de_ingreso": "numero de ingresos"}),
x = 'ingreso_mes_año',
y = 'numero de ingresos',
color = 'disease_name',
title = 'Ingresos por enfermedad',
width = 1500, height = 800
)
fig.update_layout(hovermode="x")
In [404]:
df=df.sort_values('fecha_de_ingreso')
df.head()
Out[404]:
In [405]:
df.columns
Out[405]:
In [406]:
df_reducido = df.loc[
~ ( df['diagnosis_name'].isnull() | df['disease_name'].isnull() ) & ( df['estancia'] != 0 )
].drop_duplicates(
subset=['historia_recodificada', 'cip_recodificado','fecha_de_ingreso'],
keep='first'
).reset_index(drop=True)
df_reducido.sample(5).T
Out[406]:
In [407]:
df_reducido.loc[df_reducido.edad < 15 , 'age_group'] = '<15'
df_reducido.loc[(df_reducido.edad >= 15) & (df_reducido.edad <= 40) , 'age_group'] = '15-39'
df_reducido.loc[df_reducido.edad > 39 , 'age_group'] = '>39'
In [408]:
fig, ax = plt.subplots(figsize=(20,8))
diagnosticos_respiratorios = [
'asma',
'neumonia bacteriana',
'enfermedad pulmonar obstructiva cronica'
]
fig = sns.violinplot(
y = df_reducido.loc[df_reducido.diagnosis_name.isin(diagnosticos_respiratorios), 'diagnosis_name'],
x = df_reducido.loc[df_reducido.diagnosis_name.isin(diagnosticos_respiratorios), 'estancia'],
cut=0,
hue = df_reducido.age_group, hue_order = ['<15','15-39','>39'],
scale = 'count', orient= 'h',
ax = ax
)
plt.ylabel('Diagnósticos')
plt.xlabel('Estancia (dÃas)')
# plt.xticks(rotation=90)
plt.tight_layout()
plt.show()
In [409]:
fig, ax = plt.subplots(figsize=(20,8))
enfermedades_respiratorias = [
'PQI 05 enfermedad pulmonar obstructiva cronica o asma en mayores',
'PQI 11 neumonia bacteriana',
'PQI 15 asma en adultos jovenes 15 39 anos',
'hospitalizacion por asma en poblacion infantil'
]
fig = sns.violinplot(
y = df_reducido.loc[df_reducido.disease_name.isin(enfermedades_respiratorias), 'disease_name'],
x = df_reducido.loc[df_reducido.disease_name.isin(enfermedades_respiratorias), 'estancia'],
cut=0,
hue = df_reducido.age_group, hue_order = ['<15','15-39','>39'],
scale = 'count', orient= 'h',
ax = ax
)
plt.ylabel('Enfermedades')
plt.xlabel('Estancia (dÃas)')
# plt.xticks(rotation=90)
plt.tight_layout()
plt.show()
In [410]:
fig, ax = plt.subplots(figsize=(20,8))
diagnosticos_respiratorios = [
'asma',
'neumonia bacteriana',
'enfermedad pulmonar obstructiva cronica'
]
fig = sns.boxplot(
y = df_reducido.loc[df_reducido.diagnosis_name.isin(diagnosticos_respiratorios), 'diagnosis_name'],
x = df_reducido.loc[df_reducido.diagnosis_name.isin(diagnosticos_respiratorios), 'estancia'],
hue = df_reducido.age_group, hue_order = ['<15','15-39','>39'],
orient= 'h',
ax = ax
)
plt.ylabel('Diagnósticos')
plt.xlabel('Estancia (dÃas)')
# plt.xticks(rotation=90)
plt.tight_layout()
plt.show()
In [411]:
fig, ax = plt.subplots(figsize=(20,8))
enfermedades_respiratorias = [
'PQI 05 enfermedad pulmonar obstructiva cronica o asma en mayores',
'PQI 11 neumonia bacteriana',
'PQI 15 asma en adultos jovenes 15 39 anos',
'hospitalizacion por asma en poblacion infantil'
]
fig = sns.boxplot(
y = df_reducido.loc[df_reducido.disease_name.isin(enfermedades_respiratorias), 'disease_name'],
x = df_reducido.loc[df_reducido.disease_name.isin(enfermedades_respiratorias), 'estancia'],
hue = df_reducido.age_group, hue_order = ['<15','15-39','>39'],
orient= 'h',
ax = ax
)
plt.ylabel('Enfermedades')
plt.xlabel('Estancia (dÃas)')
# plt.xticks(rotation=90)
plt.tight_layout()
plt.show()
DF Definitivo¶
In [ ]:
df_num_ingresos_diagnostico=df.groupby(['año_ingreso','mes_ingreso','diagnosis_name'])['admission_date'].count().reset_index().rename(columns={"admission_date": "numero_de_ingresos"})
In [ ]:
df_num_ingresos=df.groupby(['año_ingreso','mes_ingreso'])['admission_date'].count().reset_index().rename(columns={"admission_date": "numero_de_ingresos"})
In [ ]:
df_num_ingresos['fecha']=df_num_ingresos.año_ingreso.astype('str')+'-'+df_num_ingresos.mes_ingreso.astype('str')
In [ ]:
df_num_ingresos.fecha=pd.to_datetime(df_num_ingresos.fecha,format='%Y-%m')
In [ ]:
df_num_ingresos=df_num_ingresos[['fecha','numero_de_ingresos']]
In [ ]:
df_num_ingresos.numero_de_ingresos=df_num_ingresos.numero_de_ingresos.shift(-1)
In [ ]:
df_num_ingresos.dropna(inplace=True)
In [ ]:
df_num_ingresos
Out[ ]:
In [ ]:
df_num_ingresos_diagnostico['mes_ano']=df_num_ingresos_diagnostico.año_ingreso.astype('str')+'-'+df_num_ingresos_diagnostico.mes_ingreso.astype('str')
In [ ]:
df_num_ingresos_diagnostico.mes_ano=pd.to_datetime(df_num_ingresos_diagnostico.mes_ano,format='%Y-%m')
In [ ]:
df_num_ingresos_diagnostico_pivot=df_num_ingresos_diagnostico.pivot('mes_ano','diagnosis_name','numero_de_ingresos').reset_index()
In [ ]:
df_magnitud.ANO=df_magnitud.ANO.astype('int')
df_magnitud.MES=df_magnitud.MES.astype('int')
In [ ]:
df_merge=df_num_ingresos.merge(df_fin,on=['fecha'],how='inner')
In [ ]:
df_merge.sample(20)
Out[ ]:
In [ ]:
df_merge.drop(['COMENTARIO','ABREVIATURA','MEDIDA','ANO','MES'],axis=1,inplace=True)
In [ ]:
df_merge.head(10)
Out[ ]:
In [ ]:
df_def=pd.pivot_table(df_merge,index='fecha',columns='PARTICULA',values='value').reset_index().merge(df_num_ingresos,on='fecha',how='left').merge(df_temperatura,on='fecha',how='left')
In [ ]:
df_def[diagnosticos[:-1]]=df_def[diagnosticos[:-1]].shift(-1)
In [ ]:
df_def['Benceno_1_mes']=df_def.Benceno.shift()
df_def['Benceno_2_meses']=df_def.Benceno.shift(2)
df_def['Benceno_3_meses']=df_def.Benceno.shift(3)
df_def['DióxidodDeAzufre_1_mes']=df_def['Dióxido de Azufre'].shift()
df_def['DióxidodDeAzufre_2_meses']=df_def['Dióxido de Azufre'].shift(2)
df_def['DióxidodDeAzufre_3_meses']=df_def['Dióxido de Azufre'].shift(3)
df_def['Ozono_1_mes']=df_def.Ozono.shift()
df_def['Ozono_2_meses']=df_def.Ozono.shift(2)
df_def['Ozono_3_meses']=df_def.Ozono.shift(3)
df_def['DióxidodDeNitrogeno_1_mes']=df_def['Dióxido de Nitrógeno'].shift()
df_def['DióxidodDeNitrogeno_2_meses']=df_def['Dióxido de Nitrógeno'].shift(2)
df_def['DióxidodDeNitrogeno_3_meses']=df_def['Dióxido de Nitrógeno'].shift(3)
df_def['MonoxidoDeCarbono_1_mes']=df_def['Monóxido de Carbono'].shift()
df_def['MonoxidoDeCarbono_2_meses']=df_def['Monóxido de Carbono'].shift(2)
df_def['MonoxidoDeCarbono_3_meses']=df_def['Monóxido de Carbono'].shift(3)
df_def['OxidosDeNitrogeno_1_mes']=df_def['Óxidos de Nitrógeno'].shift()
df_def['OxidosDeNitrogeno_2_meses']=df_def['Óxidos de Nitrógeno'].shift(2)
df_def['OxidosDeNitrogeno_3_meses']=df_def['Óxidos de Nitrógeno'].shift(3)
df_def['Particulas10_1_mes']=df_def['PartÃculas menor 10 μm'].shift()
df_def['Particulas10_2_meses']=df_def['PartÃculas menor 10 μm'].shift(2)
df_def['Particulas10_3_meses']=df_def['PartÃculas menor 10 μm'].shift(3)
df_def['Particulas2.5_1_mes']=df_def['PartÃculas menor 2.5 μm'].shift()
df_def['Particulas2.5_2_meses']=df_def['PartÃculas menor 2.5 μm'].shift(2)
df_def['Particulas2.5_3_meses']=df_def['PartÃculas menor 2.5 μm'].shift(3)
df_def['ingresos_año_anterior']=df_def.numero_de_ingresos.shift(12)
df_def['mes']=df_def.fecha.dt.month
In [3]:
df_def=pd.read_csv('data/df_definitivo.csv',parse_dates=['fecha'])
df_def_diagnosticos=pd.read_csv('data/df_definitivo_por_diagnosticos.csv',parse_dates=['fecha'])
EDA sobre df definitivo¶
Obtenemos la matriz de correlación de las variables de interés (están comentadas algunas para que la matriz no quede excesivamente grande)
In [4]:
corr_df_def = df_def_diagnosticos[[
'Benceno',
'Dióxido de Azufre',
'Dióxido de Nitrógeno',
'Monóxido de Carbono',
'Ozono',
'PartÃculas menor 10 μm',
'PartÃculas menor 2.5 μm',
'Óxidos de Nitrógeno',
'enfermedad pulmonar obstructiva cronica',
'asma',
'neumonia bacteriana',
'temperatura_media_mes',
'humedad_relativa',
'precipitaciones',
'presion_atmosferica_mes',
'Benceno_1_mes',
'Benceno_2_meses',
'Benceno_3_meses',
'DióxidodDeAzufre_1_mes',
'DióxidodDeAzufre_2_meses',
'DióxidodDeAzufre_3_meses',
'Ozono_1_mes',
'Ozono_2_meses',
'Ozono_3_meses',
'DióxidodDeNitrogeno_1_mes',
'DióxidodDeNitrogeno_2_meses',
'DióxidodDeNitrogeno_3_meses',
'MonoxidoDeCarbono_1_mes',
'MonoxidoDeCarbono_2_meses',
'MonoxidoDeCarbono_3_meses',
'OxidosDeNitrogeno_1_mes',
'OxidosDeNitrogeno_2_meses',
'OxidosDeNitrogeno_3_meses',
'Particulas10_1_mes',
'Particulas10_2_meses',
'Particulas10_3_meses',
'Particulas2.5_1_mes',
'Particulas2.5_2_meses',
'Particulas2.5_3_meses',
'nb_año_anterior',
'asma_año_anterior',
# 'mes',
'epoc_año_anterior'
]].corr()
In [5]:
targets = ['enfermedad pulmonar obstructiva cronica', 'asma', 'neumonia bacteriana']
plt.subplots(figsize=(corr_df_def.shape[0]+10,4))
sns.heatmap(
corr_df_def.loc[
targets,
corr_df_def.columns[~ corr_df_def.columns.isin(targets)]
],
cmap="YlGnBu",
annot=True
)
Out[5]:
Modelos por número de ingresos totales¶
XGBoost¶
In [ ]:
columns_X=['Benceno','Dióxido de Azufre','Dióxido de Nitrógeno','Monóxido de Carbono','Ozono','PartÃculas menor 10 μm','PartÃculas menor 2.5 μm','Óxidos de Nitrógeno',
'Benceno_1_mes', 'Benceno_2_meses' ,'Benceno_3_meses', 'DióxidodDeAzufre_1_mes', 'DióxidodDeAzufre_2_meses', 'DióxidodDeAzufre_3_meses',
'Ozono_1_mes', 'Ozono_2_meses', 'Ozono_3_meses', 'DióxidodDeNitrogeno_1_mes', 'DióxidodDeNitrogeno_2_meses', 'DióxidodDeNitrogeno_3_meses',
'MonoxidoDeCarbono_1_mes', 'MonoxidoDeCarbono_2_meses', 'MonoxidoDeCarbono_3_meses', 'OxidosDeNitrogeno_1_mes', 'OxidosDeNitrogeno_2_meses',
'OxidosDeNitrogeno_3_meses', 'Particulas10_1_mes', 'Particulas10_2_meses', 'Particulas10_3_meses','Particulas2.5_1_mes', 'Particulas2.5_2_meses',
'Particulas2.5_3_meses','temperatura_media_mes','precipitaciones','presion_atmosferica_mes','humedad_relativa','ingresos_año_anterior','mes']
columns_y='numero_de_ingresos'
limite='2014-12-01'
X_train=df_def[ (~df_def[columns_y].isna()) & (df_def.fecha<=limite)][columns_X][12:]
y_train=df_def[ (~df_def[columns_y].isna()) & (df_def.fecha<=limite)][columns_y][12:]
X_test=df_def[ (~df_def[columns_y].isna()) & (df_def.fecha>limite)][columns_X]
y_test=df_def[ (~df_def[columns_y].isna()) & (df_def.fecha>limite)][columns_y]
In [ ]:
df_num_ingresos=df_def[['fecha','numero_de_ingresos']][12:]
In [ ]:
param_grid = {
'n_estimators': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
model1=XGBRegressor()
In [ ]:
cv = RandomizedSearchCV(model1, param_grid, cv = 5, verbose = True, n_jobs = -1)
In [ ]:
cv.fit(X_train, y_train)
Out[ ]:
In [ ]:
cv.best_score_,cv.best_params_
Out[ ]:
In [ ]:
t = time.perf_counter()
final_model_xgb = XGBRegressor(**cv.best_params_)
final_model_xgb.fit( X_train, y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_xgb.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
preds0=final_model_xgb.predict(X_train)
preds1=final_model_xgb.predict(X_test)
In [ ]:
rmse_xgb_train,mae_xgb_train,mape_xgb_train=np.sqrt(mean_squared_error(preds0,y_train)),mean_absolute_error(preds0,y_train),mean_absolute_percentage_error(preds0,y_train)
rmse_xgb_test, mae_xgb_test,mape_xgb_test=np.sqrt(mean_squared_error(preds1,y_test)),mean_absolute_error(preds1,y_test),mean_absolute_percentage_error(preds1,y_test)
In [ ]:
errores_xgb_train=[ mae_xgb_train,mape_xgb_train,rmse_xgb_train]
errores_xgb_test=[ mae_xgb_test,mape_xgb_test,rmse_xgb_test]
In [ ]:
explainer = shap.TreeExplainer(final_model_xgb)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
df_num_ingresos['predicciones_xgb']=final_model_xgb.predict(pd.concat([X_train,X_test]).sort_index())
CatBoost¶
In [ ]:
model2=CatBoostRegressor()
In [ ]:
param_grid = {
'iterations': [10,20,30,40],
'depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv = RandomizedSearchCV(model2, param_grid, cv = 5, verbose = True, n_jobs = -1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_score_,cv.best_params_
Out[ ]:
In [ ]:
final_model_cb=CatBoostRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_cb.fit(X_train,y_train,eval_set=[(X_train,y_train),(X_test,y_test)],early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_cb.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
pred3=final_model_cb.predict(X_train)
pred2=final_model_cb.predict(X_test)
In [ ]:
mae_cb_train,mape_cb_train,rmse_cb_train=mean_absolute_error(pred3,y_train),mean_absolute_percentage_error(pred3,y_train),np.sqrt(mean_squared_error(pred3,y_train))
mae_cb_test,mape_cb_test,rmse_cb_test=mean_absolute_error(pred2,y_test),mean_absolute_percentage_error(pred2,y_test),np.sqrt(mean_squared_error(pred2,y_test))
In [ ]:
errores_cb_train=[mae_cb_train,mape_cb_train,rmse_cb_train]
errores_cb_test=[mae_cb_test,mape_cb_test,rmse_cb_test]
In [ ]:
explainer = shap.TreeExplainer(final_model_cb)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
df_num_ingresos['predicciones_cb']=final_model_cb.predict(pd.concat([X_train,X_test]).sort_index())
LightGBM¶
In [ ]:
model3=LGBMRegressor()
In [ ]:
param_grid = {
'num_iterations': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model3,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_lgb=LGBMRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_lgb.fit(X_train,y_train, eval_set=[(X_train,y_train),(X_test,y_test)],early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_lgb.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
pred5=final_model_lgb.predict(X_train)
pred4=final_model_lgb.predict(X_test)
In [ ]:
mae_lgb_train,mape_lgb_train,rmse_lgb_train=mean_absolute_error(pred5,y_train),mean_absolute_percentage_error(pred5,y_train),np.sqrt(mean_squared_error(pred5,y_train))
mae_lgb_test,mape_lgb_test,rmse_lgb_test=mean_absolute_error(pred4,y_test),mean_absolute_percentage_error(pred4,y_test),np.sqrt(mean_squared_error(pred4,y_test))
In [ ]:
errores_lgb_train=[mae_lgb_train,mape_lgb_train,rmse_lgb_train]
errores_lgb_test=[mae_lgb_test,mape_lgb_test,rmse_lgb_test]
In [ ]:
explainer = shap.TreeExplainer(final_model_lgb)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
df_errores_train=pd.DataFrame({'errores_xgb':errores_xgb_train,'errores_cb':errores_cb_train,'errores_lgb':errores_lgb_train},index=['MAE','MAPE','RMSE'])
df_errores_test=pd.DataFrame({'errores_xgb':errores_xgb_test,'errores_cb':errores_cb_test,'errores_lgb':errores_lgb_test},index=['MAE','MAPE','RMSE'])
In [ ]:
df_num_ingresos['predicciones_lgbm']=final_model_lgb.predict(pd.concat([X_train,X_test]).sort_index())
In [ ]:
px.line(df_num_ingresos,x='fecha',y=['predicciones_lgbm','predicciones_cb','predicciones_xgb','numero_de_ingresos'])
In [ ]:
df_errores_train
Out[ ]:
In [ ]:
df_errores_test
Out[ ]:
Estimación por diagnosticos: Enfermedad pulmonar obstructiva crónica (epoc), asma (a) y neumonia bacteriana (nb)¶
XGBoost para enfermedad pulmonar obstructiva cronica¶
In [ ]:
columns_X=['Benceno','Dióxido de Azufre','Dióxido de Nitrógeno','Monóxido de Carbono','Ozono','PartÃculas menor 10 μm','PartÃculas menor 2.5 μm','Óxidos de Nitrógeno',
'Benceno_1_mes', 'Benceno_2_meses' ,'Benceno_3_meses', 'DióxidodDeAzufre_1_mes', 'DióxidodDeAzufre_2_meses', 'DióxidodDeAzufre_3_meses',
'Ozono_1_mes', 'Ozono_2_meses', 'Ozono_3_meses', 'DióxidodDeNitrogeno_1_mes', 'DióxidodDeNitrogeno_2_meses', 'DióxidodDeNitrogeno_3_meses',
'MonoxidoDeCarbono_1_mes', 'MonoxidoDeCarbono_2_meses', 'MonoxidoDeCarbono_3_meses', 'OxidosDeNitrogeno_1_mes', 'OxidosDeNitrogeno_2_meses',
'OxidosDeNitrogeno_3_meses', 'Particulas10_1_mes', 'Particulas10_2_meses', 'Particulas10_3_meses','Particulas2.5_1_mes', 'Particulas2.5_2_meses',
'Particulas2.5_3_meses','temperatura_media_mes','precipitaciones','presion_atmosferica_mes','humedad_relativa','epoc_año_anterior','mes']
columns_y='enfermedad pulmonar obstructiva cronica'
limite='2014-12-01'
X_train=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha<=limite)][columns_X][12:]
y_train=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha<=limite)][columns_y][12:]
X_test=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha>limite)][columns_X]
y_test=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha>limite)][columns_y]
In [ ]:
df_num_ingresos_epoc=df_def_diagnosticos[['fecha','enfermedad pulmonar obstructiva cronica']][12:]
In [ ]:
df_num_ingresos_epoc.dropna(inplace=True)
In [ ]:
model1=XGBRegressor()
In [ ]:
param_grid = {
'n_estimators': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model1,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_xgb_epoc=XGBRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_xgb_epoc.fit( X_train,
y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_xgb_epoc.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_xgb_epoc)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred0_epoc=final_model_xgb_epoc.predict(X_train)
pred1_epoc=final_model_xgb_epoc.predict(X_test)
In [ ]:
mae_xgb_epoc_train,mape_xgb_epoc_train,rmse_xgb_epoc_train=mean_absolute_error(pred0_epoc,y_train),mean_absolute_percentage_error(pred0_epoc,y_train),np.sqrt(mean_squared_error(pred0_epoc,y_train))
mae_xgb_epoc_test,mape_xgb_epoc_test,rmse_xgb_epoc_test=mean_absolute_error(pred1_epoc,y_test),mean_absolute_percentage_error(pred1_epoc,y_test),np.sqrt(mean_squared_error(pred1_epoc,y_test))
In [ ]:
errores_xgb_epoc_train=[mae_xgb_epoc_train,mape_xgb_epoc_train,rmse_xgb_epoc_train]
errores_xgb_epoc_test=[mae_xgb_epoc_test,mape_xgb_epoc_test,rmse_xgb_epoc_test]
In [ ]:
df_num_ingresos_epoc['predicciones_xgb']=final_model_xgb_epoc.predict(pd.concat([X_train,X_test]).sort_index())
Catboost para enfermedad pulmonar obstructiva cronica¶
In [ ]:
model2=CatBoostRegressor()
In [ ]:
param_grid = {
'iterations': [10, 20,30,40],
'depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model2,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_cb_epoc=CatBoostRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_cb_epoc.fit(X_train,y_train,eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_cb_epoc.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_cb_epoc)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred3_epoc=final_model_cb_epoc.predict(X_train)
pred2_epoc=final_model_cb_epoc.predict(X_test)
In [ ]:
mae_cb_epoc_train,mape_cb_epoc_train,rmse_cb_epoc_train=mean_absolute_error(pred3_epoc,y_train),mean_absolute_percentage_error(pred3_epoc,y_train),np.sqrt(mean_squared_error(pred3_epoc,y_train))
mae_cb_epoc_test,mape_cb_epoc_test,rmse_cb_epoc_test=mean_absolute_error(pred2_epoc,y_test),mean_absolute_percentage_error(pred2_epoc,y_test),np.sqrt(mean_squared_error(pred2_epoc,y_test))
In [ ]:
errores_cb_epoc_train=[mae_cb_epoc_train,mape_cb_epoc_train,rmse_cb_epoc_train]
errores_cb_epoc_test=[mae_cb_epoc_test,mape_cb_epoc_test,rmse_cb_epoc_test]
In [ ]:
df_num_ingresos_epoc['predicciones_cb']=final_model_cb_epoc.predict(pd.concat([X_train,X_test]).sort_index())
LightGBM para enfermedad pulmonar obstructiva cronica¶
In [ ]:
model3=LGBMRegressor()
In [ ]:
param_grid = {
'num_iterations': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model3,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_lgb_epoc=LGBMRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_lgb_epoc.fit(X_train,y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_lgb_epoc.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_lgb_epoc)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred5_epoc=final_model_lgb_epoc.predict(X_train)
pred4_epoc=final_model_lgb_epoc.predict(X_test)
In [ ]:
mae_lgb_epoc_train,mape_lgb_epoc_train,rmse_lgb_epoc_train=mean_absolute_error(pred5_epoc,y_train),mean_absolute_percentage_error(pred5_epoc,y_train),np.sqrt(mean_squared_error(pred5_epoc,y_train))
mae_lgb_epoc_test,mape_lgb_epoc_test,rmse_lgb_epoc_test=mean_absolute_error(pred4_epoc,y_test),mean_absolute_percentage_error(pred4_epoc,y_test),np.sqrt(mean_squared_error(pred4_epoc,y_test))
In [ ]:
errores_lgb_epoc_train=[mae_lgb_epoc_train,mape_lgb_epoc_train,rmse_lgb_epoc_train]
errores_lgb_epoc_test=[mae_lgb_epoc_test,mape_lgb_epoc_test,rmse_lgb_epoc_test]
In [ ]:
df_errores_epoc_train=pd.DataFrame({'errores_xgb_epoc':errores_xgb_epoc_train,'errores_cb_epoc':errores_cb_epoc_train,'errores_lgb_epoc':errores_lgb_epoc_train},index=['MAE','MAPE','RMSE'])
df_errores_epoc_test=pd.DataFrame({'errores_xgb_epoc':errores_xgb_epoc_test,'errores_cb_epoc':errores_cb_epoc_test,'errores_lgb_epoc':errores_lgb_epoc_test},index=['MAE','MAPE','RMSE'])
In [ ]:
df_errores_epoc_train
Out[ ]:
In [ ]:
df_errores_epoc_test
Out[ ]:
In [ ]:
df_num_ingresos_epoc['predicciones_lgbm']=final_model_lgb_epoc.predict(pd.concat([X_train,X_test]).sort_index())
In [ ]:
px.line(df_num_ingresos_epoc,x='fecha',y=['enfermedad pulmonar obstructiva cronica','predicciones_xgb', 'predicciones_cb', 'predicciones_lgbm'])
XGBoost para neumonia bacteriana¶
In [ ]:
columns_X=['Benceno','Dióxido de Azufre','Dióxido de Nitrógeno','Monóxido de Carbono','Ozono','PartÃculas menor 10 μm','PartÃculas menor 2.5 μm','Óxidos de Nitrógeno',
'Benceno_1_mes', 'Benceno_2_meses' ,'Benceno_3_meses', 'DióxidodDeAzufre_1_mes', 'DióxidodDeAzufre_2_meses', 'DióxidodDeAzufre_3_meses',
'Ozono_1_mes', 'Ozono_2_meses', 'Ozono_3_meses', 'DióxidodDeNitrogeno_1_mes', 'DióxidodDeNitrogeno_2_meses', 'DióxidodDeNitrogeno_3_meses',
'MonoxidoDeCarbono_1_mes', 'MonoxidoDeCarbono_2_meses', 'MonoxidoDeCarbono_3_meses', 'OxidosDeNitrogeno_1_mes', 'OxidosDeNitrogeno_2_meses',
'OxidosDeNitrogeno_3_meses', 'Particulas10_1_mes', 'Particulas10_2_meses', 'Particulas10_3_meses','Particulas2.5_1_mes', 'Particulas2.5_2_meses',
'Particulas2.5_3_meses','temperatura_media_mes','precipitaciones','presion_atmosferica_mes','humedad_relativa','nb_año_anterior','mes']
columns_y='neumonia bacteriana'
limite='2014-12-01'
X_train=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha<=limite)][columns_X][12:]
y_train=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha<=limite)][columns_y][12:]
X_test=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha>limite)][columns_X]
y_test=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha>limite)][columns_y]
In [ ]:
df_num_ingresos_nb=df_def_diagnosticos[['fecha','neumonia bacteriana']][12:]
In [ ]:
df_num_ingresos_nb.dropna(inplace=True)
In [ ]:
model1=XGBRegressor()
In [ ]:
param_grid = {
'n_estimators': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model1,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_xgb_nb=XGBRegressor(random_state=42, min_sample_split = 20,**cv.best_params_)
t=time.perf_counter()
final_model_xgb_nb.fit( X_train,
y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_xgb_nb.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_xgb_nb)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred0_nb=final_model_xgb_nb.predict(X_train)
pred1_nb=final_model_xgb_nb.predict(X_test)
In [ ]:
mae_xgb_nb_train,mape_xgb_nb_train,rmse_xgb_nb_train=mean_absolute_error(pred0_nb,y_train),mean_absolute_percentage_error(pred0_nb,y_train),np.sqrt(mean_squared_error(pred0_nb,y_train))
mae_xgb_nb_test,mape_xgb_nb_test,rmse_xgb_nb_test=mean_absolute_error(pred1_nb,y_test),mean_absolute_percentage_error(pred1_nb,y_test),np.sqrt(mean_squared_error(pred1_nb,y_test))
In [ ]:
errores_xgb_nb_train=[mae_xgb_nb_train,mape_xgb_nb_train,rmse_xgb_nb_train]
errores_xgb_nb_test=[mae_xgb_nb_test,mape_xgb_nb_test,rmse_xgb_nb_test]
In [ ]:
df_num_ingresos_nb['predicciones_xgb']=final_model_xgb_nb.predict(pd.concat([X_train,X_test]).sort_index())
Catboost para neumonia bacteriana¶
In [ ]:
model2=CatBoostRegressor()
In [ ]:
param_grid = {
'iterations': [10, 20,30,40],
'depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model2,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_cb_nb=CatBoostRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_cb_nb.fit(X_train,y_train,eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_cb_nb.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_cb_nb)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred3_nb=final_model_cb_nb.predict(X_train)
pred2_nb=final_model_cb_nb.predict(X_test)
In [ ]:
mae_cb_nb_train,mape_cb_nb_train,rmse_cb_nb_train=mean_absolute_error(pred3_nb,y_train),mean_absolute_percentage_error(pred3_nb,y_train),np.sqrt(mean_squared_error(pred3_nb,y_train))
mae_cb_nb_test,mape_cb_nb_test,rmse_cb_nb_test=mean_absolute_error(pred2_nb,y_test),mean_absolute_percentage_error(pred2_nb,y_test),np.sqrt(mean_squared_error(pred2_nb,y_test))
In [ ]:
errores_cb_nb_train=[mae_cb_nb_train,mape_cb_nb_train,rmse_cb_nb_train]
errores_cb_nb_test=[mae_cb_nb_test,mape_cb_nb_test,rmse_cb_nb_test]
In [ ]:
df_num_ingresos_nb['predicciones_cb']=final_model_cb_nb.predict(pd.concat([X_train,X_test]).sort_index())
LightGBM para neumonia bacteriana¶
In [ ]:
model3=LGBMRegressor()
In [ ]:
param_grid = {
'num_iterations': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model3,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_lgb_nb=LGBMRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_lgb_nb.fit(X_train,y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_lgb_nb.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_lgb_nb)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred5_nb=final_model_lgb_nb.predict(X_train)
pred4_nb=final_model_lgb_nb.predict(X_test)
In [ ]:
mae_lgb_nb_train,mape_lgb_nb_train,rmse_lgb_nb_train=mean_absolute_error(pred5_nb,y_train),mean_absolute_percentage_error(pred5_nb,y_train),np.sqrt(mean_squared_error(pred5_nb,y_train))
mae_lgb_nb_test,mape_lgb_nb_test,rmse_lgb_nb_test=mean_absolute_error(pred4_nb,y_test),mean_absolute_percentage_error(pred4_nb,y_test),np.sqrt(mean_squared_error(pred4_nb,y_test))
In [ ]:
errores_lgb_nb_train=[mae_lgb_nb_train,mape_lgb_nb_train,rmse_lgb_nb_train]
errores_lgb_nb_test=[mae_lgb_nb_test,mape_lgb_nb_test,rmse_lgb_nb_test]
In [ ]:
df_errores_nb_train=pd.DataFrame({'errores_xgb_nb':errores_xgb_nb_train,'errores_cb_nb':errores_cb_nb_train,'errores_lgb_nb':errores_lgb_nb_train},index=['MAE','MAPE','RMSE'])
df_errores_nb_test=pd.DataFrame({'errores_xgb_nb':errores_xgb_nb_test,'errores_cb_nb':errores_cb_nb_test,'errores_lgb_nb':errores_lgb_nb_test},index=['MAE','MAPE','RMSE'])
In [ ]:
df_errores_nb_train
Out[ ]:
In [ ]:
df_errores_nb_test
Out[ ]:
In [ ]:
df_num_ingresos_nb['predicciones_lgbm']=final_model_lgb_nb.predict(pd.concat([X_train,X_test]).sort_index())
In [ ]:
px.line(df_num_ingresos_nb,x='fecha',y=['neumonia bacteriana','predicciones_lgbm','predicciones_cb','predicciones_xgb'])
XGBoost para asma¶
In [ ]:
columns_X=['Benceno','Dióxido de Azufre','Dióxido de Nitrógeno','Monóxido de Carbono','Ozono','PartÃculas menor 10 μm','PartÃculas menor 2.5 μm','Óxidos de Nitrógeno',
'Benceno_1_mes', 'Benceno_2_meses' ,'Benceno_3_meses', 'DióxidodDeAzufre_1_mes', 'DióxidodDeAzufre_2_meses', 'DióxidodDeAzufre_3_meses',
'Ozono_1_mes', 'Ozono_2_meses', 'Ozono_3_meses', 'DióxidodDeNitrogeno_1_mes', 'DióxidodDeNitrogeno_2_meses', 'DióxidodDeNitrogeno_3_meses',
'MonoxidoDeCarbono_1_mes', 'MonoxidoDeCarbono_2_meses', 'MonoxidoDeCarbono_3_meses', 'OxidosDeNitrogeno_1_mes', 'OxidosDeNitrogeno_2_meses',
'OxidosDeNitrogeno_3_meses', 'Particulas10_1_mes', 'Particulas10_2_meses', 'Particulas10_3_meses','Particulas2.5_1_mes', 'Particulas2.5_2_meses',
'Particulas2.5_3_meses','temperatura_media_mes','precipitaciones','presion_atmosferica_mes','humedad_relativa','asma_año_anterior','mes']
columns_y='asma'
limite='2014-12-01'
X_train=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha<=limite)][columns_X][12:]
y_train=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha<=limite)][columns_y][12:]
X_test=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha>limite)][columns_X][:12]
y_test=df_def_diagnosticos[ (~df_def_diagnosticos[columns_y].isna()) & (df_def_diagnosticos.fecha>limite)][columns_y][:12]
In [ ]:
df_num_ingresos_a=df_def_diagnosticos[['fecha','asma']][13:]
In [ ]:
df_num_ingresos_a.dropna(inplace=True)
In [ ]:
model1=XGBRegressor()
In [ ]:
param_grid = {
'n_estimators': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model1,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_xgb_a=XGBRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_xgb_a.fit( X_train,
y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_xgb_a.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_xgb_a)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred0_a=final_model_xgb_a.predict(X_train)
pred1_a=final_model_xgb_a.predict(X_test)
In [ ]:
mae_xgb_a_train,mape_xgb_a_train,rmse_xgb_a_train=mean_absolute_error(pred0_a,y_train),mean_absolute_percentage_error(pred0_a,y_train),np.sqrt(mean_squared_error(pred0_a,y_train))
mae_xgb_a_test,mape_xgb_a_test,rmse_xgb_a_test=mean_absolute_error(pred1_a,y_test),mean_absolute_percentage_error(pred1_a,y_test),np.sqrt(mean_squared_error(pred1_a,y_test))
In [ ]:
errores_xgb_a_train=[mae_xgb_a_train,mape_xgb_a_train,rmse_xgb_a_train]
errores_xgb_a_test=[mae_xgb_a_test,mape_xgb_a_test,rmse_xgb_a_test]
In [ ]:
df_num_ingresos_a['predicciones_xgb']=final_model_xgb_a.predict(pd.concat([X_train,X_test]).sort_index())
CatBoost para asma¶
In [ ]:
model2=CatBoostRegressor()
In [ ]:
param_grid = {
'iterations': [10, 20,30,40],
'depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model2,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_cb_a=CatBoostRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_cb_a.fit(X_train,y_train,eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=10)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_cb_a.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_cb_a)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred3_a=final_model_cb_a.predict(X_train)
pred2_a=final_model_cb_a.predict(X_test)
In [ ]:
mae_cb_a_train,mape_cb_a_train,rmse_cb_a_train=mean_absolute_error(pred3_a,y_train),mean_absolute_percentage_error(pred3_a,y_train),np.sqrt(mean_squared_error(pred3_a,y_train))
mae_cb_a_test,mape_cb_a_test,rmse_cb_a_test=mean_absolute_error(pred2_a,y_test),mean_absolute_percentage_error(pred2_a,y_test),np.sqrt(mean_squared_error(pred2_a,y_test))
In [ ]:
errores_cb_a_train=[mae_cb_a_train,mape_cb_a_train,rmse_cb_a_train]
errores_cb_a_test=[mae_cb_a_test,mape_cb_a_test,rmse_cb_a_test]
In [ ]:
df_num_ingresos_a['predicciones_cb']=final_model_cb_a.predict(pd.concat([X_train,X_test]).sort_index())
LightGBM para asma¶
In [ ]:
model3=LGBMRegressor()
In [ ]:
param_grid = {
'num_iterations': [10, 20,30,40],
'max_depth': [2,3,4,5],
'learning_rate':[0.01,0.001,0.1]
}
In [ ]:
cv=RandomizedSearchCV(model3,param_grid,cv=5,n_jobs=-1)
In [ ]:
cv.fit(X_train,y_train)
Out[ ]:
In [ ]:
cv.best_params_,cv.best_score_
Out[ ]:
In [ ]:
final_model_lgb_a=LGBMRegressor(**cv.best_params_)
t=time.perf_counter()
final_model_lgb_a.fit(X_train, y_train,
eval_set=[(X_train, y_train), (X_test, y_test)],
early_stopping_rounds=5)
Out[ ]:
In [ ]:
elapsed_time = datetime.timedelta(seconds=(time.perf_counter() - t))
print('Tiempo de entrenamiento:', elapsed_time)
In [ ]:
feature_importance = final_model_lgb_a.feature_importances_
sorted_idx = np.argsort(feature_importance)
fig = plt.figure(figsize=(12, 6))
plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')
plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])
plt.title('Feature Importance')
Out[ ]:
In [ ]:
explainer = shap.TreeExplainer(final_model_lgb_a)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
In [ ]:
pred5_a=final_model_lgb_a.predict(X_train)
pred4_a=final_model_lgb_a.predict(X_test)
In [ ]:
mae_lgb_a_train,mape_lgb_a_train,rmse_lgb_a_train=mean_absolute_error(pred5_a,y_train),mean_absolute_percentage_error(pred5_a,y_train),np.sqrt(mean_squared_error(pred5_a,y_train))
mae_lgb_a_test,mape_lgb_a_test,rmse_lgb_a_test=mean_absolute_error(pred4_a,y_test),mean_absolute_percentage_error(pred4_a,y_test),np.sqrt(mean_squared_error(pred4_a,y_test))
In [ ]:
errores_lgb_a_train=[mae_lgb_a_train,mape_lgb_a_train,rmse_lgb_a_train]
errores_lgb_a_test=[mae_lgb_a_test,mape_lgb_a_test,rmse_lgb_a_test]
In [ ]:
df_errores_a_train=pd.DataFrame({'errores_xgb_a':errores_xgb_a_train,'errores_cb_a':errores_cb_a_train,'errores_lgb_a':errores_lgb_a_train},index=['MAE','MAPE','RMSE'])
df_errores_a_test=pd.DataFrame({'errores_xgb_a':errores_xgb_a_test,'errores_cb_a':errores_cb_a_test,'errores_lgb_a':errores_lgb_a_test},index=['MAE','MAPE','RMSE'])
In [ ]:
df_errores_a_train
Out[ ]:
In [ ]:
df_errores_a_test
Out[ ]:
In [ ]:
df_num_ingresos_a['predicciones_lgbm']=final_model_lgb_a.predict(pd.concat([X_train,X_test]).sort_index())
In [ ]:
px.line(df_num_ingresos_a,x='fecha',y=['asma','predicciones_lgbm','predicciones_cb','predicciones_xgb'])
Diferencia porcentual entre datos reales y modelos¶
In [ ]:
df_comparativa_real_modelos = df_num_ingresos.merge(
df_num_ingresos_a,
how = 'outer',
on = 'fecha',
suffixes=('','_asma')
).merge(
df_num_ingresos_nb,
how = 'outer',
on = 'fecha',
suffixes=('','_neum_bact')
).merge(
df_num_ingresos_epoc,
how = 'outer',
on = 'fecha',
suffixes=('','_enf_pulm_obstr_cron')
)
df_comparativa_real_modelos.sample(5).T
In [ ]:
campos = [
['difer_porcent_xgb', 'predicciones_xgb', 'numero_de_ingresos'],
['difer_porcent_cb', 'predicciones_cb', 'numero_de_ingresos'],
['difer_porcent_lgbm', 'predicciones_lgbm', 'numero_de_ingresos'],
['difer_porcent_xgb_asma', 'predicciones_xgb_asma', 'asma'],
['difer_porcent_cb_asma', 'predicciones_cb_asma', 'asma'],
['difer_porcent_lgbm_asma', 'predicciones_lgbm_asma', 'asma'],
['difer_porcent_xgb_neum_bact', 'predicciones_xgb_neum_bact', 'neumonia bacteriana'],
['difer_porcent_cb_neum_bact', 'predicciones_cb_neum_bact', 'neumonia bacteriana'],
['difer_porcent_lgbm_neum_bact', 'predicciones_lgbm_neum_bact', 'neumonia bacteriana'],
['difer_porcent_xgb_enf_pulm_obstr_cron', 'predicciones_xgb_enf_pulm_obstr_cron', 'enfermedad pulmonar obstructiva cronica'],
['difer_porcent_cb_enf_pulm_obstr_cron', 'predicciones_cb_enf_pulm_obstr_cron', 'enfermedad pulmonar obstructiva cronica'],
['difer_porcent_lgbm_enf_pulm_obstr_cron', 'predicciones_lgbm_enf_pulm_obstr_cron', 'enfermedad pulmonar obstructiva cronica'],
]
for x in campos:
df_comparativa_real_modelos[x[0]] = round(
np.abs( ( ( df_comparativa_real_modelos[x[1]] - df_comparativa_real_modelos[x[2]] ) / df_comparativa_real_modelos[x[2]] ) * 100 ),
2
)
df_comparativa_real_modelos.sample(5).T
In [ ]:
df_comparativa_real_modelos['year'] = df_comparativa_real_modelos.fecha.dt.year
df_comparativa_real_modelos['month'] = df_comparativa_real_modelos.fecha.dt.month
In [ ]:
fig = plt.subplots(figsize=(12,12))
_ = sns.heatmap(
df_comparativa_real_modelos[[
'month',
'difer_porcent_xgb',
'difer_porcent_cb',
'difer_porcent_lgbm',
'difer_porcent_xgb_asma',
'difer_porcent_cb_asma',
'difer_porcent_lgbm_asma',
'difer_porcent_xgb_neum_bact',
'difer_porcent_cb_neum_bact',
'difer_porcent_lgbm_neum_bact',
'difer_porcent_xgb_enf_pulm_obstr_cron',
'difer_porcent_cb_enf_pulm_obstr_cron',
'difer_porcent_lgbm_enf_pulm_obstr_cron',
]].groupby(['month']).mean([
'difer_porcent_xgb',
'difer_porcent_cb',
'difer_porcent_lgbm',
'difer_porcent_xgb_asma',
'difer_porcent_cb_asma',
'difer_porcent_lgbm_asma',
'difer_porcent_xgb_neum_bact',
'difer_porcent_cb_neum_bact',
'difer_porcent_lgbm_neum_bact',
'difer_porcent_xgb_enf_pulm_obstr_cron',
'difer_porcent_cb_enf_pulm_obstr_cron',
'difer_porcent_lgbm_enf_pulm_obstr_cron',
]).T,
annot=True,
cmap='seismic'
)
